# -*- coding: utf-8 -*-
import scrapy

import re
import json
import time
import random
import logging
from selenium import webdriver
from copy import copy, deepcopy
from scrapy.selector import Selector
from FDC_spider.settings import USER_AGENTS, PROXY_URL
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem
from FDC_spider.constants import SHH_EG_STATUS_DICT, SHH_RM_STATUS_DICT, SHH_RM_FLAT_DICT

logger = logging.getLogger(__name__)


class ShanghaiSpider(scrapy.Spider):
    name = 'shanghai'
    allowed_domains = ['fangdi.com.cn']
    """
    GET请求：获取cookies
    POST请求：获取数据
    """
    start_urls = ['http://www.fangdi.com.cn/service/freshHouse/getHosueList.action']  # POST  项目列表url
    project_li_url = 'http://www.fangdi.com.cn/new_house/new_house_list.html'  # GET   项目列表url
    project_temp_url = 'http://www.fangdi.com.cn/new_house/new_house_detail.html?project_id={}'  # GET   项目详情url
    project_url = 'http://www.fangdi.com.cn/service/freshHouse/queryProjectById.actin'  # POST  项目详情url
    cert_bd_url = 'http://www.fangdi.com.cn/service/freshHouse/queryStartUnit.action'  # POST  许可证和楼栋url
    rm_li_temp_url = 'http://www.fangdi.com.cn/new_house/more_info.html?project_id={}&building_id={}&start_id={}'  # GET   房号列表url
    rm_li_url = 'http://www.fangdi.com.cn/service/freshHouse/getMoreInfo.action'  # POST   房号列表url

    def start_requests(self):
        data = dict(
            districtID='',
            dicRegionID='',
            stateID='',
            houseAreaID='',
            dicAvgpriceID='',
            dicPositionID='',
            houseTypeID='',
            address='',
            openingID='',
            projectName='',
            currentPage='1',
        )
        url = ShanghaiSpider.project_li_url
        yield scrapy.FormRequest(
            ShanghaiSpider.start_urls[0],
            formdata=data,
            cookies=self.get_cookies_dict(url),
            headers=self.get_headers(url),
            meta=dict(current_page=1),
        )

    def parse(self, response):
        """
        解析项目列表页
        :param response:
        :return:
        """
        current_page = copy(response.meta['current_page'])
        try:
            total_page_str = response.xpath("//span[@class='page_total']/i/text()").extract_first()
            total_page_li = re.findall(r'\d+', total_page_str)
            total_page = total_page_li[0]
        except Exception as e:
            logging.warning('第{}页总页数获取失败  error_msg:{}'.format(current_page, e))
        else:
            print('>>>>>>>>第{}/{}页<<<<<<<<'.format(current_page, total_page))
            try:
                response = self.format_pt_html(response.body.decode())
            except Exception as e:
                logging.error('第{}页项目列表html清洗失败  error_msg:{}'.format(current_page, e))
            else:
                tr_li = response.xpath("//tr")
                if len(tr_li):
                    for tr in tr_li[1:4]:
                        item_eg = FdcEstateGuidItem()
                        status_num = tr.xpath("./td[1]/text()").extract_first()
                        item_eg['projectStatus'] = SHH_EG_STATUS_DICT.get(status_num, '其它')
                        item_eg['projectName'] = tr.xpath("./td[2]/a/text()").extract_first()
                        pt_id_str = tr.xpath("./td[2]/a/@onclick").extract_first()
                        item_eg['projectAddress'] = tr.xpath("./td[3]/text()").extract_first()
                        item_eg['totalRoomNo'] = tr.xpath("./td[4]/text()").extract_first()
                        item_eg['estateTotalArea'] = tr.xpath("./td[5]/text()").extract_first()
                        item_eg['districtName'] = tr.xpath("./td[6]/text()").extract_first()
                        item_eg['projectId'] = self.get_project_id(pt_id_str)
                        if item_eg['projectId']:
                            # 构造POST请求参数
                            data = dict(projectID=item_eg['projectId'])
                            # 项目详情url
                            url = ShanghaiSpider.project_temp_url.format(item_eg['projectId'])
                            yield scrapy.FormRequest(
                                ShanghaiSpider.project_url,
                                formdata=data,
                                cookies=self.get_cookies_dict(url),
                                headers=self.get_headers(url),
                                callback=self.parse_project,
                                meta=dict(item_eg=deepcopy(item_eg)),
                            )
                        else:
                            logging.error('第{}页-{} 项目id解析为空'.format(current_page, item_eg['projectName']))
                else:
                    logging.error('第{}页项目列表获取为空'.format(current_page))

            # 翻页
            # if int(current_page) < int(total_page):
            if int(current_page) < 1:
                # 构造POST参数
                data = dict(
                    districtID='',
                    dicRegionID='',
                    stateID='',
                    houseAreaID='',
                    dicAvgpriceID='',
                    dicPositionID='',
                    houseTypeID='',
                    address='',
                    openingID='',
                    projectName='',
                    currentPage=str(int(current_page) + 1),
                )
                url = ShanghaiSpider.project_li_url
                yield scrapy.FormRequest(
                    ShanghaiSpider.start_urls[0],
                    formdata=data,
                    cookies=self.get_cookies_dict(url),
                    headers=self.get_headers(url),
                    meta=dict(current_page=str(int(current_page) + 1)),
                )

    def parse_project(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        resp_dict = json.loads(response.body.decode())
        if resp_dict.get('project', None):
            item_eg['developerName'] = resp_dict['project'].get('name', None)
            item_eg['totalSaleableRoomNum'] = resp_dict['project'].get('leaving_num', None)
            item_eg['totalSaleableArea'] = resp_dict['project'].get('leaving_area', None)
            item_eg['totalResidenceNum'] = resp_dict['project'].get('z_num', None)
            item_eg['totalResidenceArea'] = resp_dict['project'].get('z_area', None)
            item_eg['totalSaleableResidenceNum'] = resp_dict['project'].get('z_leaving_num', None)
            item_eg['totalSaleableResidenceArea'] = resp_dict['project'].get('z_leaving_area', None)
            # 构造POST请求参数
            data = dict(projectID=item_eg['projectId'])
            # 项目详情url
            url = ShanghaiSpider.project_temp_url.format(item_eg['projectId'])
            yield scrapy.FormRequest(
                ShanghaiSpider.cert_bd_url,
                formdata=data,
                cookies=self.get_cookies_dict(url),
                headers=self.get_headers(url),
                callback=self.parse_permit_building_li,
                meta=dict(item_eg=deepcopy(item_eg)),
            )
        else:
            logging.error(
                '{}-{}-{} 项目详情信息获取失败'.format(item_eg['districtName'], item_eg['projectName'], item_eg['projectId']))

    def parse_permit_building_li(self, response):
        """
        获取许可证列表和楼栋列表数据
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        resp_dict = json.loads(response.body.decode())
        # 许可证列表
        permit_li = resp_dict.get('startUnitList', [])
        # 楼栋列表
        bd_li = resp_dict.get('priceInformationList', [])
        if len(permit_li) and len(bd_li) and len(permit_li) == len(bd_li):
            for i, permit_dict in enumerate(permit_li[0:1]):
                item_eg['preSalePermit'] = permit_dict.get('presell_desc', None)
                item_eg['permitId'] = permit_dict.get('presell_id', None)
                item_eg['permitTotalRoomNum'] = permit_dict.get('num', None)
                item_eg['permitResidenceNum'] = permit_dict.get('sell_num', None)
                item_eg['permitSaleableRoomNum'] = permit_dict.get('leaving_num', None)
                item_eg['certDate'] = permit_dict.get('start_date', None)
                item_eg['certDate'] = item_eg['certDate'] if item_eg['certDate'] else None
                item_eg['permitStatus'] = self.get_permit_status(permit_dict)
                # yield item_eg
                print(item_eg)
                # 获取与当前许可证对应的楼栋列表
                sub_bd_li = bd_li[i]
                if sub_bd_li and len(sub_bd_li):
                    for bd_dict in sub_bd_li[0:1]:
                        item_bd = FdcBuildingItem()
                        item_bd['districtName'] = item_eg['districtName']
                        item_bd['projectName'] = item_eg['projectName']
                        item_bd['projectId'] = item_eg['projectId']
                        item_bd['preSalePermit'] = item_eg['preSalePermit']
                        item_bd['permitId'] = item_eg['permitId']
                        start_id = bd_dict.get('start_id', None)
                        item_bd['blockName'] = bd_dict.get('building_name', None)
                        item_bd['buildingId'] = bd_dict.get('building_id', None)
                        item_bd['totalRoomNo'] = bd_dict.get('num', None)
                        item_bd['saleableRoomNo'] = bd_dict.get('leaving_num', None)
                        item_bd['hignRefPrice'] = bd_dict.get('hign_refprice', None)
                        item_bd['lowRefPrice'] = bd_dict.get('low_refprice', None)
                        if item_bd['hignRefPrice'] and item_bd['lowRefPrice']:
                            item_bd['referencePrice'] = (int(item_bd['hignRefPrice']) + int(item_bd['lowRefPrice'])) / 2
                        else:
                            item_bd['referencePrice'] = bd_dict.get('reference_price', None)
                        # yield item_bd
                        print(item_bd)
                        # 构造POST请求参数
                        data = dict(
                            buildingID=item_bd['buildingId'],
                            startID=start_id,
                        )
                        # 房号列表url
                        url = ShanghaiSpider.rm_li_temp_url.format(item_bd['projectId'], item_bd['buildingId'],
                                                                   start_id)
                        # yield scrapy.FormRequest(
                        #     ShanghaiSpider.rm_li_url,
                        #     formdata=data,
                        #     cookies=self.get_cookies_dict(url),
                        #     headers=self.get_headers(url),
                        #     callback=self.parse_room_li,
                        #     meta=dict(item_bd=deepcopy(item_bd)),
                        # )
        else:
            logging.error(
                '{}-{}-{}  许可证、楼栋列表获取错误，许可证列表长度{}，楼栋列表长度{}'.format(item_eg['districtName'], item_eg['projectName'],
                                                                   item_eg['projectId'], len(permit_li), len(bd_li)))

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        resp_dict = json.loads(response.body.decode())
        if resp_dict.get('moreInfoList', {}):
            # 按楼层分组
            for floor, rm_li in resp_dict['moreInfoList'].items():
                if len(rm_li):
                    for rm_dict in rm_li:
                        if len(rm_dict.keys()) > 7 and rm_dict.get('room_number', None):
                            item_rm = FdcRoomItem()
                            item_rm['districtName'] = item_bd['districtName']
                            item_rm['projectName'] = item_bd['projectName']
                            item_rm['projectId'] = item_bd['projectId']
                            item_rm['preSalePermit'] = item_bd['preSalePermit']
                            item_rm['permitId'] = item_bd['permitId']
                            item_rm['blockName'] = item_bd['blockName']
                            item_rm['buildingId'] = item_bd['buildingId']
                            item_rm['roomFloor'] = floor
                            item_rm['roomNo'] = rm_dict.get('room_number', None)
                            item_rm['realBuildArea'] = rm_dict.get('flarea', None)
                            item_rm['realInnerArea'] = rm_dict.get('priv_flarea', None)
                            status = rm_dict.get('status', None)
                            item_rm['saleStatus'] = SHH_RM_STATUS_DICT.get(str(status), None) if status else None
                            flat_style = rm_dict.get('flat_style', None)
                            item_rm['roomStructure'] = SHH_RM_FLAT_DICT.get(str(flat_style),
                                                                            None) if flat_style else None
                            item_rm['roomUse'] = self.get_room_use(rm_dict)
                            if not item_rm['roomUse']:
                                logging.warning(
                                    '{}-{}-{}-{}-{}  房号用途获取为空'.format(item_rm['districtName'], item_rm['projectName'],
                                                                      item_rm['preSalePermit'], item_rm['blockName'],
                                                                      item_rm['roomNo']))
                            # yield item_rm
                            print(item_rm)
        else:
            logging.error('{}-{}-{}-{}  楼层和房号信息获取为空'.format(item_bd['districtName'], item_bd['projectName'],
                                                            item_bd['preSalePermit'], item_bd['blockName']))

    @staticmethod
    def get_cookies_dict(_url):
        """
        获取cookies字典
        :param _url:
        :return:
        """

        def get_ua(): return random.choice(USER_AGENTS)

        options = webdriver.ChromeOptions()
        # options.add_argument('--headless')  # 不提供可视化页面
        options.add_experimental_option('excludeSwitches', ['enable-automation'])  # 开发者模式
        options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})  # 不加载图片
        options.add_argument('--disable-gpu')  # 规避bug
        options.add_argument('--disable-infobars')  # 禁用浏览器正在被自动化程序控制的提示
        options.add_argument('--incognito')  # 隐身模式（无痕模式）
        options.add_argument('--no-sandbox')  # 解决DevToolsActivePort文件不存在的报错
        options.add_argument('User-Agent={}'.format(get_ua))  # 设置UA
        # options.add_argument('--proxy-server=http://ip:port')  # 设置无账号密码的代理
        # proxyauth_plugin_path = dict(
        #     proxy_host='host',
        #     proxy_port='port',
        #     proxy_username="username",
        #     proxy_password="password"
        # )
        # options.add_extension(proxyauth_plugin_path)  # 设置有账号密码的代理
        driver = webdriver.Chrome(options=options)
        # driver.set_window_size(400, 300)  # 设置窗口大小
        # driver.maximize_window()  # 窗口最大化
        driver.get(url=_url)
        time.sleep(3)
        cookies_dict = {cookie['name']: cookie['value'] for cookie in driver.get_cookies()}
        driver.close()
        return cookies_dict

    @staticmethod
    def get_headers(_url):
        """
        构造请求头
        :param _url:
        :return:
        """
        headers_dict = dict(
            Referer=_url,
            Host='www.fangdi.com.cn',
            Origin='http://www.fangdi.com.cn',
        )
        return headers_dict

    @staticmethod
    def format_pt_html(_html):
        """
        清洗html
        :param _html:
        :return:
        """
        regex_1 = re.compile(r'(<table.*table>)', re.S)
        regex_2 = re.compile(r'\n')
        regex_3 = re.compile(r'\\/')
        html = regex_3.sub('/', regex_2.sub('', regex_1.findall(_html)[0]))
        return Selector(text=html)

    @staticmethod
    def get_project_id(_cont):
        """
        提取项目id
        :param _cont:
        :return:
        """
        regex = re.compile(r'houseDetail[(]\'(\w+)\'[)]')
        pt_id_li = regex.findall(_cont)
        return pt_id_li[0] if len(pt_id_li) else None

    @staticmethod
    def get_permit_status(_dict):
        """
        获取许可证销售状态
        :param _dict:
        :return:
        """
        if _dict.get('startFlag', None) != 1:
            if _dict.get('leaving_num', None) > 0:
                if _dict.get('issalable', None) == 2:
                    return '暂停销售'
                else:
                    return '在售'
            else:
                return '售完'
        else:
            return '即将开盘'

    @staticmethod
    def get_room_use(_dict):
        """
        获取房号用途
        :param _dict:
        :return:
        """
        if _dict['land_type'] and int(_dict['land_type']) == 1:
            return '公寓'
        elif _dict['land_type'] and int(_dict['land_type']) == 2:
            return '办公楼'
        elif _dict['land_type'] and int(_dict['land_type']) == 3:
            if _dict['land_use'] and int(_dict['land_use']) == 41:
                return '商场'
            elif _dict['land_use'] and int(_dict['land_use']) == 43:
                return '店铺'
        elif _dict['land_type'] and int(_dict['land_type']) == 4:
            return '其它'


if __name__ == '__main__':
    a1 = dict(
        a=1,
        b=2,
        c=3,
    )